{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# mlflow-energyforecast\n",
    "\n",
    "This is a showcase for ML Flow capabilities, based on the article\n",
    "http://the-odd-dataguy.com/be-more-efficient-to-produce-ml-models-with-mlflow\n",
    "and a github https://github.com/jeanmidevacc/mlflow-energyforecast\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting pandas\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/12/d1/a6502c2f5c15b50f5dd579fc1c52b47edf6f2e9f682aed917dd7565b3e60/pandas-1.0.0-cp36-cp36m-manylinux1_x86_64.whl (10.1MB)\n",
      "\u001b[K     |████████████████████████████████| 10.1MB 3.2MB/s eta 0:00:01\n",
      "\u001b[?25hRequirement already satisfied, skipping upgrade: numpy>=1.13.3 in ./.local/lib/python3.6/site-packages (from pandas) (1.18.1)\n",
      "Requirement already satisfied, skipping upgrade: python-dateutil>=2.6.1 in /usr/local/lib/python3.6/dist-packages (from pandas) (2.8.0)\n",
      "Requirement already satisfied, skipping upgrade: pytz>=2017.2 in /usr/local/lib/python3.6/dist-packages (from pandas) (2019.2)\n",
      "Requirement already satisfied, skipping upgrade: six>=1.5 in /usr/lib/python3/dist-packages (from python-dateutil>=2.6.1->pandas) (1.11.0)\n",
      "Installing collected packages: pandas\n",
      "  Found existing installation: pandas 0.25.3\n",
      "    Uninstalling pandas-0.25.3:\n",
      "      Successfully uninstalled pandas-0.25.3\n",
      "Successfully installed pandas-1.0.0\n",
      "\u001b[33mWARNING: You are using pip version 19.1.1, however version 20.0.2 is available.\n",
      "You should consider upgrading via the 'pip install --upgrade pip' command.\u001b[0m\n",
      "Collecting mlflow\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/65/33/5fe1559f7eb95e1fa2077df747ada7fd225045bad4e76bcdb53605e4b937/mlflow-1.6.0.tar.gz (15.9MB)\n",
      "\u001b[K     |████████████████████████████████| 15.9MB 3.0MB/s eta 0:00:01\n",
      "\u001b[?25hRequirement already satisfied, skipping upgrade: alembic in ./.local/lib/python3.6/site-packages (from mlflow) (1.3.2)\n",
      "Requirement already satisfied, skipping upgrade: click>=7.0 in /usr/local/lib/python3.6/dist-packages (from mlflow) (7.0)\n",
      "Requirement already satisfied, skipping upgrade: cloudpickle in ./.local/lib/python3.6/site-packages (from mlflow) (1.1.1)\n",
      "Requirement already satisfied, skipping upgrade: databricks-cli>=0.8.7 in ./.local/lib/python3.6/site-packages (from mlflow) (0.9.1)\n",
      "Requirement already satisfied, skipping upgrade: requests>=2.17.3 in /usr/local/lib/python3.6/dist-packages (from mlflow) (2.22.0)\n",
      "Requirement already satisfied, skipping upgrade: six>=1.10.0 in /usr/lib/python3/dist-packages (from mlflow) (1.11.0)\n",
      "Requirement already satisfied, skipping upgrade: Flask in ./.local/lib/python3.6/site-packages (from mlflow) (1.1.1)\n",
      "Requirement already satisfied, skipping upgrade: numpy in ./.local/lib/python3.6/site-packages (from mlflow) (1.18.1)\n",
      "Requirement already satisfied, skipping upgrade: pandas in ./.local/lib/python3.6/site-packages (from mlflow) (1.0.0)\n",
      "Requirement already satisfied, skipping upgrade: python-dateutil in /usr/local/lib/python3.6/dist-packages (from mlflow) (2.8.0)\n",
      "Requirement already satisfied, skipping upgrade: protobuf>=3.6.0 in /usr/local/lib/python3.6/dist-packages (from mlflow) (3.8.0)\n",
      "Requirement already satisfied, skipping upgrade: gitpython>=2.1.0 in ./.local/lib/python3.6/site-packages (from mlflow) (3.0.5)\n",
      "Requirement already satisfied, skipping upgrade: pyyaml in /usr/local/lib/python3.6/dist-packages (from mlflow) (5.1.2)\n",
      "Requirement already satisfied, skipping upgrade: querystring_parser in ./.local/lib/python3.6/site-packages (from mlflow) (1.2.4)\n",
      "Requirement already satisfied, skipping upgrade: simplejson in ./.local/lib/python3.6/site-packages (from mlflow) (3.17.0)\n",
      "Requirement already satisfied, skipping upgrade: docker>=4.0.0 in /usr/local/lib/python3.6/dist-packages (from mlflow) (4.0.2)\n",
      "Requirement already satisfied, skipping upgrade: entrypoints in /usr/local/lib/python3.6/dist-packages (from mlflow) (0.3)\n",
      "Requirement already satisfied, skipping upgrade: sqlparse in ./.local/lib/python3.6/site-packages (from mlflow) (0.3.0)\n",
      "Requirement already satisfied, skipping upgrade: sqlalchemy in ./.local/lib/python3.6/site-packages (from mlflow) (1.3.12)\n",
      "Requirement already satisfied, skipping upgrade: gorilla in ./.local/lib/python3.6/site-packages (from mlflow) (0.3.0)\n",
      "Requirement already satisfied, skipping upgrade: prometheus-flask-exporter in ./.local/lib/python3.6/site-packages (from mlflow) (0.12.1)\n",
      "Requirement already satisfied, skipping upgrade: gunicorn in ./.local/lib/python3.6/site-packages (from mlflow) (20.0.4)\n",
      "Requirement already satisfied, skipping upgrade: Mako in ./.local/lib/python3.6/site-packages (from alembic->mlflow) (1.1.0)\n",
      "Requirement already satisfied, skipping upgrade: python-editor>=0.3 in ./.local/lib/python3.6/site-packages (from alembic->mlflow) (1.0.4)\n",
      "Requirement already satisfied, skipping upgrade: configparser>=0.3.5 in ./.local/lib/python3.6/site-packages (from databricks-cli>=0.8.7->mlflow) (4.0.2)\n",
      "Requirement already satisfied, skipping upgrade: tabulate>=0.7.7 in /usr/local/lib/python3.6/dist-packages (from databricks-cli>=0.8.7->mlflow) (0.8.3)\n",
      "Requirement already satisfied, skipping upgrade: chardet<3.1.0,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests>=2.17.3->mlflow) (3.0.4)\n",
      "Requirement already satisfied, skipping upgrade: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests>=2.17.3->mlflow) (2019.9.11)\n",
      "Requirement already satisfied, skipping upgrade: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests>=2.17.3->mlflow) (1.24.3)\n",
      "Requirement already satisfied, skipping upgrade: idna<2.9,>=2.5 in /usr/lib/python3/dist-packages (from requests>=2.17.3->mlflow) (2.6)\n",
      "Requirement already satisfied, skipping upgrade: Jinja2>=2.10.1 in /usr/local/lib/python3.6/dist-packages (from Flask->mlflow) (2.10.1)\n",
      "Requirement already satisfied, skipping upgrade: itsdangerous>=0.24 in ./.local/lib/python3.6/site-packages (from Flask->mlflow) (1.1.0)\n",
      "Requirement already satisfied, skipping upgrade: Werkzeug>=0.15 in /usr/local/lib/python3.6/dist-packages (from Flask->mlflow) (0.15.4)\n",
      "Requirement already satisfied, skipping upgrade: pytz>=2017.2 in /usr/local/lib/python3.6/dist-packages (from pandas->mlflow) (2019.2)\n",
      "Requirement already satisfied, skipping upgrade: setuptools in /usr/local/lib/python3.6/dist-packages (from protobuf>=3.6.0->mlflow) (41.0.1)\n",
      "Requirement already satisfied, skipping upgrade: gitdb2>=2.0.0 in ./.local/lib/python3.6/site-packages (from gitpython>=2.1.0->mlflow) (2.0.6)\n",
      "Requirement already satisfied, skipping upgrade: websocket-client>=0.32.0 in /usr/local/lib/python3.6/dist-packages (from docker>=4.0.0->mlflow) (0.56.0)\n",
      "Requirement already satisfied, skipping upgrade: prometheus-client in /usr/local/lib/python3.6/dist-packages (from prometheus-flask-exporter->mlflow) (0.7.1)\n",
      "Requirement already satisfied, skipping upgrade: MarkupSafe>=0.9.2 in /usr/local/lib/python3.6/dist-packages (from Mako->alembic->mlflow) (1.1.1)\n",
      "Requirement already satisfied, skipping upgrade: smmap2>=2.0.0 in ./.local/lib/python3.6/site-packages (from gitdb2>=2.0.0->gitpython>=2.1.0->mlflow) (2.0.5)\n",
      "Building wheels for collected packages: mlflow\n",
      "  Building wheel for mlflow (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25h  Stored in directory: /home/jovyan/.cache/pip/wheels/46/4e/83/e58b14b6d2d494783e31690de9572c5777882f675f480374b6\n",
      "Successfully built mlflow\n",
      "Installing collected packages: mlflow\n",
      "  Found existing installation: mlflow 1.5.0\n",
      "    Uninstalling mlflow-1.5.0:\n",
      "      Successfully uninstalled mlflow-1.5.0\n",
      "\u001b[33m  WARNING: The script mlflow is installed in '/home/jovyan/.local/bin' which is not on PATH.\n",
      "  Consider adding this directory to PATH or, if you prefer to suppress this warning, use --no-warn-script-location.\u001b[0m\n",
      "Successfully installed mlflow-1.6.0\n",
      "\u001b[33mWARNING: You are using pip version 19.1.1, however version 20.0.2 is available.\n",
      "You should consider upgrading via the 'pip install --upgrade pip' command.\u001b[0m\n",
      "Requirement already up-to-date: joblib in ./.local/lib/python3.6/site-packages (0.14.1)\n",
      "\u001b[33mWARNING: You are using pip version 19.1.1, however version 20.0.2 is available.\n",
      "You should consider upgrading via the 'pip install --upgrade pip' command.\u001b[0m\n",
      "Requirement already up-to-date: numpy in ./.local/lib/python3.6/site-packages (1.18.1)\n",
      "\u001b[33mWARNING: You are using pip version 19.1.1, however version 20.0.2 is available.\n",
      "You should consider upgrading via the 'pip install --upgrade pip' command.\u001b[0m\n",
      "Requirement already up-to-date: scipy in ./.local/lib/python3.6/site-packages (1.4.1)\n",
      "Requirement already satisfied, skipping upgrade: numpy>=1.13.3 in ./.local/lib/python3.6/site-packages (from scipy) (1.18.1)\n",
      "\u001b[33mWARNING: You are using pip version 19.1.1, however version 20.0.2 is available.\n",
      "You should consider upgrading via the 'pip install --upgrade pip' command.\u001b[0m\n",
      "Requirement already up-to-date: scikit-learn in ./.local/lib/python3.6/site-packages (0.22.1)\n",
      "Requirement already satisfied, skipping upgrade: numpy>=1.11.0 in ./.local/lib/python3.6/site-packages (from scikit-learn) (1.18.1)\n",
      "Requirement already satisfied, skipping upgrade: scipy>=0.17.0 in ./.local/lib/python3.6/site-packages (from scikit-learn) (1.4.1)\n",
      "Requirement already satisfied, skipping upgrade: joblib>=0.11 in ./.local/lib/python3.6/site-packages (from scikit-learn) (0.14.1)\n",
      "\u001b[33mWARNING: You are using pip version 19.1.1, however version 20.0.2 is available.\n",
      "You should consider upgrading via the 'pip install --upgrade pip' command.\u001b[0m\n",
      "Collecting boto3\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/d5/57/e9675a5a8d0ee586594ff19cb9a601334fbf24fa2fb29052d2a900ee5d23/boto3-1.11.9-py2.py3-none-any.whl (128kB)\n",
      "\u001b[K     |████████████████████████████████| 133kB 3.5MB/s eta 0:00:01\n",
      "\u001b[?25hCollecting botocore<1.15.0,>=1.14.9 (from boto3)\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/64/4c/b0b0d3b6f84a05f9135051b56d3eb8708012a289c4b82ee21c8c766f47b5/botocore-1.14.9-py2.py3-none-any.whl (5.9MB)\n",
      "\u001b[K     |████████████████████████████████| 5.9MB 11.6MB/s eta 0:00:01\n",
      "\u001b[?25hRequirement already satisfied, skipping upgrade: jmespath<1.0.0,>=0.7.1 in ./.local/lib/python3.6/site-packages (from boto3) (0.9.4)\n",
      "Requirement already satisfied, skipping upgrade: s3transfer<0.4.0,>=0.3.0 in ./.local/lib/python3.6/site-packages (from boto3) (0.3.0)\n",
      "Requirement already satisfied, skipping upgrade: python-dateutil<3.0.0,>=2.1 in /usr/local/lib/python3.6/dist-packages (from botocore<1.15.0,>=1.14.9->boto3) (2.8.0)\n",
      "Requirement already satisfied, skipping upgrade: docutils<0.16,>=0.10 in ./.local/lib/python3.6/site-packages (from botocore<1.15.0,>=1.14.9->boto3) (0.15.2)\n",
      "Requirement already satisfied, skipping upgrade: urllib3<1.26,>=1.20 in /usr/local/lib/python3.6/dist-packages (from botocore<1.15.0,>=1.14.9->boto3) (1.24.3)\n",
      "Requirement already satisfied, skipping upgrade: six>=1.5 in /usr/lib/python3/dist-packages (from python-dateutil<3.0.0,>=2.1->botocore<1.15.0,>=1.14.9->boto3) (1.11.0)\n",
      "Installing collected packages: botocore, boto3\n",
      "  Found existing installation: botocore 1.14.4\n",
      "    Uninstalling botocore-1.14.4:\n",
      "      Successfully uninstalled botocore-1.14.4\n",
      "  Found existing installation: boto3 1.11.4\n",
      "    Uninstalling boto3-1.11.4:\n",
      "      Successfully uninstalled boto3-1.11.4\n",
      "Successfully installed boto3-1.11.9 botocore-1.14.9\n",
      "\u001b[33mWARNING: You are using pip version 19.1.1, however version 20.0.2 is available.\n",
      "You should consider upgrading via the 'pip install --upgrade pip' command.\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "!pip install pandas --upgrade --user\n",
    "!pip install mlflow --upgrade --user\n",
    "!pip install joblib --upgrade --user\n",
    "!pip install numpy --upgrade --user \n",
    "!pip install scipy --upgrade --user \n",
    "!pip install scikit-learn --upgrade --user\n",
    "!pip install boto3 --upgrade --user"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import time\n",
    "import json\n",
    "import os\n",
    "from joblib import Parallel, delayed\n",
    "\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import scipy\n",
    "\n",
    "from sklearn.model_selection import train_test_split, KFold\n",
    "from sklearn.metrics import mean_squared_error, mean_absolute_error, r2_score, explained_variance_score\n",
    "from sklearn.exceptions import ConvergenceWarning\n",
    "\n",
    "import mlflow\n",
    "import mlflow.sklearn\n",
    "from  mlflow.tracking import MlflowClient\n",
    "\n",
    "from warnings import simplefilter\n",
    "simplefilter(action='ignore', category = FutureWarning)\n",
    "simplefilter(action='ignore', category = ConvergenceWarning)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Ensure Minio access\n",
    "os.environ['MLFLOW_S3_ENDPOINT_URL'] = 'http://minio-service.kubeflow.svc.cluster.local:9000'\n",
    "os.environ['AWS_ACCESS_KEY_ID'] = 'minio'\n",
    "os.environ['AWS_SECRET_ACCESS_KEY'] = 'minio123'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data preparation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Collect the data \n",
    "df_nationalconsumption_electricity_daily = pd.read_csv(\"https://raw.githubusercontent.com/jeanmidevacc/mlflow-energyforecast/master/data/rtu_data.csv\")\n",
    "df_nationalconsumption_electricity_daily.set_index([\"day\"], inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Size of the training set :  1081\n",
      "Size of the testing set :  233\n"
     ]
    }
   ],
   "source": [
    "# Prepare the training set and the testing set\n",
    "df_trainvalidate_energyconsumption = df_nationalconsumption_electricity_daily[df_nationalconsumption_electricity_daily[\"datastatus\"] == \"Définitif\"]\n",
    "del df_trainvalidate_energyconsumption[\"datastatus\"]\n",
    "\n",
    "df_test_energyconsumption = df_nationalconsumption_electricity_daily[df_nationalconsumption_electricity_daily[\"datastatus\"] == \"Consolidé\"]\n",
    "del df_test_energyconsumption[\"datastatus\"]\n",
    "\n",
    "print(\"Size of the training set : \",len(df_trainvalidate_energyconsumption))\n",
    "print(\"Size of the testing set : \",len(df_test_energyconsumption))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Output to predict :  dailyconsumption\n",
      "Inputs for the prediction :  ['weekday', 'week', 'month', 'year', 'avg_min_temperature', 'avg_max_temperature', 'avg_mean_temperature', 'wavg_min_temperature', 'wavg_max_temperature', 'wavg_mean_temperature', 'is_holiday']\n"
     ]
    }
   ],
   "source": [
    "# Define the inputs and the output\n",
    "output = \"dailyconsumption\"\n",
    "allinputs = list(df_trainvalidate_energyconsumption.columns)\n",
    "allinputs.remove(output)\n",
    "\n",
    "print(\"Output to predict : \", output)\n",
    "print(\"Inputs for the prediction : \", allinputs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Build different set of featurws for the model\n",
    "possible_inputs = {\n",
    "    \"all\" : allinputs,\n",
    "    \"only_allday_inputs\" : [\"weekday\", \"month\", \"is_holiday\", \"week\"],\n",
    "    \"only_allweatheravg_inputs\" : [\"avg_min_temperature\", \"avg_max_temperature\", \"avg_mean_temperature\",\"wavg_min_temperature\", \"wavg_max_temperature\", \"wavg_mean_temperature\"],\n",
    "    \"only_meanweather_inputs_avg\" : [\"avg_mean_temperature\"],\n",
    "    \"only_meanweather_inputs_wavg\" : [\"wavg_mean_temperature\"],\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Prepare the output of the model\n",
    "array_output_train = np.array(df_trainvalidate_energyconsumption[output])\n",
    "array_output_test = np.array(df_test_energyconsumption[output])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# connect to remote server\n",
    "remote_server_uri = \"http://mlflowserver.kubeflow.svc.cluster.local:5000\"\n",
    "mlflow.set_tracking_uri(remote_server_uri)\n",
    "# Launch the experiment on mlflow\n",
    "experiment_name = \"electricityconsumption-forecast\"\n",
    "mlflow.set_experiment(experiment_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define the evaluation function that will do the computation of the different metrics of accuracy (RMSE,MAE,R2)\n",
    "def evaluation_model(y_test, y_pred):\n",
    "\n",
    "    rmse = np.sqrt(mean_squared_error(y_test, y_pred))\n",
    "    mae = mean_absolute_error(y_test, y_pred)\n",
    "    r2 = r2_score(y_test, y_pred)\n",
    "\n",
    "    metrics = {\n",
    "        \"rmse\" : rmse,\n",
    "        \"r2\" : r2,\n",
    "        \"mae\" : mae,\n",
    "    }\n",
    "    \n",
    "    return metrics"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# KNN regressor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.neighbors import KNeighborsRegressor\n",
    "\n",
    "def train_knnmodel(parameters, inputs, tags, log = False):\n",
    "    with mlflow.start_run(nested = True):\n",
    "        \n",
    "        # Prepare the data\n",
    "        array_inputs_train = np.array(df_trainvalidate_energyconsumption[inputs])\n",
    "        array_inputs_test = np.array(df_test_energyconsumption[inputs])\n",
    "        \n",
    "        \n",
    "        # Build the model\n",
    "        tic = time.time()\n",
    "        model = KNeighborsRegressor(parameters[\"nbr_neighbors\"], weights = parameters[\"weight_method\"])\n",
    "        model.fit(array_inputs_train, array_output_train)\n",
    "        duration_training = time.time() - tic\n",
    "\n",
    "        # Make the prediction\n",
    "        tic1 = time.time()\n",
    "        prediction = model.predict(array_inputs_test)\n",
    "        duration_prediction = time.time() - tic1\n",
    "\n",
    "        # Evaluate the model prediction\n",
    "        metrics = evaluation_model(array_output_test, prediction)\n",
    "\n",
    "        # Log in the console\n",
    "        if log:\n",
    "            print(f\"KNN regressor:\")\n",
    "            print(parameters)\n",
    "            print(metrics)\n",
    "\n",
    "        # Log in mlflow (parameter)\n",
    "        mlflow.log_params(parameters)\n",
    "\n",
    "        # Log in mlflow (metrics)\n",
    "        metrics[\"duration_training\"] = duration_training\n",
    "        metrics[\"duration_prediction\"] = duration_prediction\n",
    "        mlflow.log_metrics(metrics)\n",
    "\n",
    "        # log in mlflow (model)\n",
    "        mlflow.sklearn.log_model(model, f\"model\")\n",
    "                \n",
    "        # Tag the model\n",
    "        mlflow.set_tags(tags)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Test the different combinations\n",
    "configurations = []\n",
    "for nbr_neighbors in [1,2,5,10]:\n",
    "    for weight_method in ['uniform','distance']:\n",
    "        for field in possible_inputs:\n",
    "            parameters = {\n",
    "                \"nbr_neighbors\" : nbr_neighbors,\n",
    "                \"weight_method\" : weight_method\n",
    "            }\n",
    "\n",
    "            tags = {\n",
    "                \"model\" : \"knn\",\n",
    "                \"inputs\" : field\n",
    "            }\n",
    "            \n",
    "            configurations.append([parameters, tags])\n",
    "\n",
    "            train_knnmodel(parameters, possible_inputs[field], tags)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# MLP regressor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.neural_network import MLPRegressor\n",
    "\n",
    "def train_mlpmodel(parameters, inputs, tags, log = False):\n",
    "    with mlflow.start_run(nested = True):\n",
    "        \n",
    "        # Prepare the data\n",
    "        array_inputs_train = np.array(df_trainvalidate_energyconsumption[inputs])\n",
    "        array_inputs_test = np.array(df_test_energyconsumption[inputs])\n",
    "        \n",
    "        # Build the model\n",
    "        tic = time.time()\n",
    "\n",
    "        model = MLPRegressor(\n",
    "            hidden_layer_sizes = parameters[\"hidden_layers\"],\n",
    "            activation = parameters[\"activation\"],\n",
    "            solver = parameters[\"solver\"],\n",
    "            max_iter = parameters[\"nbr_iteration\"],\n",
    "            random_state = 0)\n",
    "        \n",
    "        model.fit(array_inputs_train, array_output_train)\n",
    "        duration_training = time.time() - tic\n",
    "\n",
    "        # Make the prediction\n",
    "        tic1 = time.time()\n",
    "        prediction = model.predict(array_inputs_test)\n",
    "        duration_prediction = time.time() - tic1\n",
    "\n",
    "        # Evaluate the model prediction\n",
    "        metrics = evaluation_model(array_output_test, prediction)\n",
    "\n",
    "        # Log in the console\n",
    "        if log:\n",
    "            print(f\"Random forest regressor:\")\n",
    "            print(parameters)\n",
    "            print(metrics)\n",
    "    \n",
    "        # Log in mlflow (parameter)\n",
    "        mlflow.log_params(parameters)\n",
    "\n",
    "        # Log in mlflow (metrics)\n",
    "        metrics[\"duration_training\"] = duration_training\n",
    "        metrics[\"duration_prediction\"] = duration_prediction\n",
    "        mlflow.log_metrics(metrics)\n",
    "\n",
    "        # log in mlflow (model)\n",
    "        mlflow.sklearn.log_model(model, f\"model\")\n",
    "        \n",
    "        # Tag the model\n",
    "        mlflow.set_tags(tags)        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "for hiddenlayers in [4,8,16]:\n",
    "    for activation in [\"identity\",\"logistic\",]:\n",
    "        for solver in [\"lbfgs\"]:\n",
    "            for nbriteration in [10,100,1000]:\n",
    "                for field in possible_inputs:\n",
    "                    parameters = {\n",
    "                        \"hidden_layers\" : hiddenlayers,\n",
    "                        \"activation\" : activation,\n",
    "                        \"solver\" : solver,\n",
    "                        \"nbr_iteration\" : nbriteration\n",
    "                    }\n",
    "\n",
    "                    tags = {\n",
    "                        \"model\" : \"mlp\",\n",
    "                        \"inputs\" : field\n",
    "                    }\n",
    "\n",
    "                    train_mlpmodel(parameters, possible_inputs[field], tags)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Use a handmade model (scipy approach)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "class PTG:\n",
    "    def __init__(self, thresholds_x0, thresholds_a, thresholds_b):\n",
    "        self.thresholds_x0 = thresholds_x0\n",
    "        self.thresholds_a = thresholds_a\n",
    "        self.thresholds_b = thresholds_b\n",
    "        \n",
    "    def get_ptgmodel(self, x, a, b, x0):\n",
    "        return np.piecewise(x, [x < x0, x >= x0], [lambda x: a*x + b , lambda x : a*x0 + b])\n",
    "        \n",
    "    def fit(self, dfx, y):\n",
    "        x = np.array(dfx)\n",
    "        \n",
    "        # Define the bounds\n",
    "        bounds_min = [thresholds_a[0], thresholds_b[0], thresholds_x0[0]]\n",
    "        bounds_max = [thresholds_a[1], thresholds_b[1], thresholds_x0[1]]\n",
    "        bounds = (bounds_min, bounds_max)\n",
    "\n",
    "        # Fit a model\n",
    "        popt, pcov = scipy.optimize.curve_fit(self.get_ptgmodel, x, y, bounds = bounds)\n",
    "\n",
    "        # Get the parameter of the model\n",
    "        a = popt[0]\n",
    "        b = popt[1]\n",
    "        x0 = popt[2]\n",
    "        \n",
    "        self.coefficients = [a, b, x0]\n",
    "        \n",
    "    def predict(self,dfx):\n",
    "        x = np.array(dfx)\n",
    "        predictions = []\n",
    "        for elt in x:\n",
    "            forecast = self.get_ptgmodel(elt, self.coefficients[0], self.coefficients[1], self.coefficients[2])\n",
    "            predictions.append(forecast)\n",
    "        return np.array(predictions)\n",
    "        \n",
    "def train_ptgmodel(parameters, inputs, tags, log = False):\n",
    "    with mlflow.start_run(nested = True):\n",
    "        \n",
    "        # Prepare the data\n",
    "        df_inputs_train = df_trainvalidate_energyconsumption[inputs[0]]\n",
    "        df_inputs_test = df_test_energyconsumption[inputs[0]]\n",
    "        \n",
    "        \n",
    "        # Build the model\n",
    "        tic = time.time()\n",
    "        \n",
    "        model = PTG(parameters[\"thresholds_x0\"], parameters[\"thresholds_a\"], parameters[\"thresholds_b\"])\n",
    "        \n",
    "        model.fit(df_inputs_train, array_output_train)\n",
    "        duration_training = time.time() - tic\n",
    "\n",
    "        # Make the prediction\n",
    "        tic1 = time.time()\n",
    "        prediction = model.predict(df_inputs_test)\n",
    "        duration_prediction = time.time() - tic1\n",
    "\n",
    "        # Evaluate the model prediction\n",
    "        metrics = evaluation_model(array_output_test, prediction)\n",
    "\n",
    "        # Log in the console\n",
    "        if log:\n",
    "            print(f\"PTG:\")\n",
    "            print(parameters)\n",
    "            print(metrics)\n",
    "    \n",
    "        # Log in mlflow (parameter)\n",
    "        mlflow.log_params(parameters)  \n",
    "\n",
    "        # Log in mlflow (metrics)\n",
    "        metrics[\"duration_training\"] = duration_training\n",
    "        metrics[\"duration_prediction\"] = duration_prediction\n",
    "        mlflow.log_metrics(metrics)\n",
    "\n",
    "        # log in mlflow (model)\n",
    "        mlflow.sklearn.log_model(model, f\"model\")\n",
    "        \n",
    "        # Tag the model\n",
    "        mlflow.set_tags(tags)           "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define the parameters of the model\n",
    "thresholds_x0 = [0, 20]\n",
    "thresholds_a = [-200000, -50000]\n",
    "thresholds_b = [1000000, 3000000]\n",
    "\n",
    "parameters = {\n",
    "    \"thresholds_x0\" : thresholds_x0,\n",
    "    \"thresholds_a\" : thresholds_a,\n",
    "    \"thresholds_b\" : thresholds_b\n",
    "}\n",
    "\n",
    "for field in [\"only_meanweather_inputs_avg\", \"only_meanweather_inputs_wavg\"]:\n",
    "    \n",
    "    tags = {\n",
    "        \"model\" : \"ptg\",\n",
    "        \"inputs\" : field\n",
    "    }\n",
    "    \n",
    "    train_ptgmodel(parameters, possible_inputs[field], tags, log = False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Evaluate mlflow results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of runs done :  272\n"
     ]
    }
   ],
   "source": [
    "# Select the run of the experiment\n",
    "df_runs = mlflow.search_runs(experiment_ids=\"0\")\n",
    "print(\"Number of runs done : \", len(df_runs))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>run_id</th>\n",
       "      <th>experiment_id</th>\n",
       "      <th>status</th>\n",
       "      <th>artifact_uri</th>\n",
       "      <th>start_time</th>\n",
       "      <th>end_time</th>\n",
       "      <th>metrics.r2</th>\n",
       "      <th>metrics.mae</th>\n",
       "      <th>metrics.duration_prediction</th>\n",
       "      <th>metrics.rmse</th>\n",
       "      <th>...</th>\n",
       "      <th>params.activation</th>\n",
       "      <th>params.nbr_iteration</th>\n",
       "      <th>params.hidden_layers</th>\n",
       "      <th>params.nbr_neighbors</th>\n",
       "      <th>params.weight_method</th>\n",
       "      <th>tags.model</th>\n",
       "      <th>tags.mlflow.source.type</th>\n",
       "      <th>tags.inputs</th>\n",
       "      <th>tags.mlflow.user</th>\n",
       "      <th>tags.mlflow.source.name</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>238</th>\n",
       "      <td>50ee6409ad3a4778bb9d8cb59034df5d</td>\n",
       "      <td>0</td>\n",
       "      <td>FINISHED</td>\n",
       "      <td>s3://mlflow/mlflow/artifacts/0/50ee6409ad3a477...</td>\n",
       "      <td>2020-01-17 18:17:47.448000+00:00</td>\n",
       "      <td>2020-01-17 18:17:47.929000+00:00</td>\n",
       "      <td>0.935956</td>\n",
       "      <td>104040.339809</td>\n",
       "      <td>0.003205</td>\n",
       "      <td>134649.399348</td>\n",
       "      <td>...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>5</td>\n",
       "      <td>distance</td>\n",
       "      <td>knn</td>\n",
       "      <td>LOCAL</td>\n",
       "      <td>all</td>\n",
       "      <td>jovyan</td>\n",
       "      <td>/usr/local/lib/python3.6/dist-packages/ipykern...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>106</th>\n",
       "      <td>614bcf7042ca465c8d86296f12ac9c09</td>\n",
       "      <td>0</td>\n",
       "      <td>FINISHED</td>\n",
       "      <td>s3://mlflow/mlflow/artifacts/0/614bcf7042ca465...</td>\n",
       "      <td>2020-01-31 15:21:29.978000+00:00</td>\n",
       "      <td>2020-01-31 15:21:30.503000+00:00</td>\n",
       "      <td>0.935956</td>\n",
       "      <td>104040.339809</td>\n",
       "      <td>0.003404</td>\n",
       "      <td>134649.399348</td>\n",
       "      <td>...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>5</td>\n",
       "      <td>distance</td>\n",
       "      <td>knn</td>\n",
       "      <td>LOCAL</td>\n",
       "      <td>all</td>\n",
       "      <td>jovyan</td>\n",
       "      <td>/usr/local/lib/python3.6/dist-packages/ipykern...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>96</th>\n",
       "      <td>b05667486f7d45779d23519eb0dbe24f</td>\n",
       "      <td>0</td>\n",
       "      <td>FINISHED</td>\n",
       "      <td>s3://mlflow/mlflow/artifacts/0/b05667486f7d457...</td>\n",
       "      <td>2020-01-31 15:21:35.424000+00:00</td>\n",
       "      <td>2020-01-31 15:21:35.922000+00:00</td>\n",
       "      <td>0.935111</td>\n",
       "      <td>105833.358681</td>\n",
       "      <td>0.002732</td>\n",
       "      <td>135534.759873</td>\n",
       "      <td>...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>10</td>\n",
       "      <td>distance</td>\n",
       "      <td>knn</td>\n",
       "      <td>LOCAL</td>\n",
       "      <td>all</td>\n",
       "      <td>jovyan</td>\n",
       "      <td>/usr/local/lib/python3.6/dist-packages/ipykern...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>228</th>\n",
       "      <td>d279d728946e4b74811203a842d79df3</td>\n",
       "      <td>0</td>\n",
       "      <td>FINISHED</td>\n",
       "      <td>s3://mlflow/mlflow/artifacts/0/d279d728946e4b7...</td>\n",
       "      <td>2020-01-17 18:17:52.555000+00:00</td>\n",
       "      <td>2020-01-17 18:17:53.029000+00:00</td>\n",
       "      <td>0.935111</td>\n",
       "      <td>105833.358681</td>\n",
       "      <td>0.002863</td>\n",
       "      <td>135534.759873</td>\n",
       "      <td>...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>10</td>\n",
       "      <td>distance</td>\n",
       "      <td>knn</td>\n",
       "      <td>LOCAL</td>\n",
       "      <td>all</td>\n",
       "      <td>jovyan</td>\n",
       "      <td>/usr/local/lib/python3.6/dist-packages/ipykern...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>111</th>\n",
       "      <td>88af21719e0a408b91448f7ddd27e84c</td>\n",
       "      <td>0</td>\n",
       "      <td>FINISHED</td>\n",
       "      <td>s3://mlflow/mlflow/artifacts/0/88af21719e0a408...</td>\n",
       "      <td>2020-01-31 15:21:27.338000+00:00</td>\n",
       "      <td>2020-01-31 15:21:27.947000+00:00</td>\n",
       "      <td>0.934465</td>\n",
       "      <td>105793.727897</td>\n",
       "      <td>0.002668</td>\n",
       "      <td>136207.422483</td>\n",
       "      <td>...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>5</td>\n",
       "      <td>uniform</td>\n",
       "      <td>knn</td>\n",
       "      <td>LOCAL</td>\n",
       "      <td>all</td>\n",
       "      <td>jovyan</td>\n",
       "      <td>/usr/local/lib/python3.6/dist-packages/ipykern...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 25 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                               run_id experiment_id    status  \\\n",
       "238  50ee6409ad3a4778bb9d8cb59034df5d             0  FINISHED   \n",
       "106  614bcf7042ca465c8d86296f12ac9c09             0  FINISHED   \n",
       "96   b05667486f7d45779d23519eb0dbe24f             0  FINISHED   \n",
       "228  d279d728946e4b74811203a842d79df3             0  FINISHED   \n",
       "111  88af21719e0a408b91448f7ddd27e84c             0  FINISHED   \n",
       "\n",
       "                                          artifact_uri  \\\n",
       "238  s3://mlflow/mlflow/artifacts/0/50ee6409ad3a477...   \n",
       "106  s3://mlflow/mlflow/artifacts/0/614bcf7042ca465...   \n",
       "96   s3://mlflow/mlflow/artifacts/0/b05667486f7d457...   \n",
       "228  s3://mlflow/mlflow/artifacts/0/d279d728946e4b7...   \n",
       "111  s3://mlflow/mlflow/artifacts/0/88af21719e0a408...   \n",
       "\n",
       "                          start_time                         end_time  \\\n",
       "238 2020-01-17 18:17:47.448000+00:00 2020-01-17 18:17:47.929000+00:00   \n",
       "106 2020-01-31 15:21:29.978000+00:00 2020-01-31 15:21:30.503000+00:00   \n",
       "96  2020-01-31 15:21:35.424000+00:00 2020-01-31 15:21:35.922000+00:00   \n",
       "228 2020-01-17 18:17:52.555000+00:00 2020-01-17 18:17:53.029000+00:00   \n",
       "111 2020-01-31 15:21:27.338000+00:00 2020-01-31 15:21:27.947000+00:00   \n",
       "\n",
       "     metrics.r2    metrics.mae  metrics.duration_prediction   metrics.rmse  \\\n",
       "238    0.935956  104040.339809                     0.003205  134649.399348   \n",
       "106    0.935956  104040.339809                     0.003404  134649.399348   \n",
       "96     0.935111  105833.358681                     0.002732  135534.759873   \n",
       "228    0.935111  105833.358681                     0.002863  135534.759873   \n",
       "111    0.934465  105793.727897                     0.002668  136207.422483   \n",
       "\n",
       "     ...  params.activation params.nbr_iteration params.hidden_layers  \\\n",
       "238  ...               None                 None                 None   \n",
       "106  ...               None                 None                 None   \n",
       "96   ...               None                 None                 None   \n",
       "228  ...               None                 None                 None   \n",
       "111  ...               None                 None                 None   \n",
       "\n",
       "    params.nbr_neighbors params.weight_method tags.model  \\\n",
       "238                    5             distance        knn   \n",
       "106                    5             distance        knn   \n",
       "96                    10             distance        knn   \n",
       "228                   10             distance        knn   \n",
       "111                    5              uniform        knn   \n",
       "\n",
       "    tags.mlflow.source.type tags.inputs tags.mlflow.user  \\\n",
       "238                   LOCAL         all           jovyan   \n",
       "106                   LOCAL         all           jovyan   \n",
       "96                    LOCAL         all           jovyan   \n",
       "228                   LOCAL         all           jovyan   \n",
       "111                   LOCAL         all           jovyan   \n",
       "\n",
       "                               tags.mlflow.source.name  \n",
       "238  /usr/local/lib/python3.6/dist-packages/ipykern...  \n",
       "106  /usr/local/lib/python3.6/dist-packages/ipykern...  \n",
       "96   /usr/local/lib/python3.6/dist-packages/ipykern...  \n",
       "228  /usr/local/lib/python3.6/dist-packages/ipykern...  \n",
       "111  /usr/local/lib/python3.6/dist-packages/ipykern...  \n",
       "\n",
       "[5 rows x 25 columns]"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Quick sorting to get the best models based on the RMSE metric\n",
    "df_runs.sort_values([\"metrics.rmse\"], ascending = True, inplace = True)\n",
    "df_runs.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'50ee6409ad3a4778bb9d8cb59034df5d'"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Get the best one\n",
    "runid_selected = df_runs.head(1)[\"run_id\"].values[0]\n",
    "runid_selected"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "celltoolbar": "Raw Cell Format",
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
